{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "from mxnet import init, gluon, nd, autograd, image\n",
    "from mxnet.gluon import nn\n",
    "import numpy as np\n",
    "import pickle as p\n",
    "import matplotlib.pyplot as plt\n",
    "from time import time\n",
    "%matplotlib inline\n",
    "ctx = mx.gpu()\n",
    "data_dir = '/home/sinyer/python/data/cifar10'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_cifar(route = data_dir+'/cifar-10-batches-py'):\n",
    "    def load_batch(filename):\n",
    "        with open(filename, 'rb')as f:\n",
    "            data_dict = p.load(f, encoding='latin1')\n",
    "            X = data_dict['data']\n",
    "            Y = data_dict['labels']\n",
    "            X = X.reshape(10000, 3, 32,32).astype(\"float\")\n",
    "            Y = np.array(Y)\n",
    "            return X, Y\n",
    "    def load_labels(filename):\n",
    "        with open(filename, 'rb') as f:\n",
    "            label_names = p.load(f, encoding='latin1')\n",
    "            names = label_names['label_names']\n",
    "            return names\n",
    "    label_names = load_labels(route + \"/batches.meta\")\n",
    "    x1, y1 = load_batch(route + \"/data_batch_1\")\n",
    "    x2, y2 = load_batch(route + \"/data_batch_2\")\n",
    "    x3, y3 = load_batch(route + \"/data_batch_3\")\n",
    "    x4, y4 = load_batch(route + \"/data_batch_4\")\n",
    "    x5, y5 = load_batch(route + \"/data_batch_5\")\n",
    "    test_pic, test_label = load_batch(route + \"/test_batch\")\n",
    "    train_pic = np.concatenate((x1, x2, x3, x4, x5))\n",
    "    train_label = np.concatenate((y1, y2, y3, y4, y5))\n",
    "    return train_pic, train_label, test_pic, test_label\n",
    "\n",
    "def accuracy(output, label):\n",
    "    return nd.mean(output.argmax(axis=1)==label).asscalar()\n",
    "\n",
    "def evaluate_accuracy(test_data, net, ctx):\n",
    "    acc = 0.\n",
    "    for data, label in test_data:\n",
    "        data = data.as_in_context(ctx)\n",
    "        label = label.as_in_context(ctx)\n",
    "        output = net(data)\n",
    "        acc = acc + accuracy(output, label)\n",
    "    return acc / len(test_data)\n",
    "\n",
    "def augment(data, auglist):\n",
    "    data = nd.pad(data, pad_width=(0,0,0,0,2,2,2,2),mode='constant',constant_value=0)\n",
    "    data = nd.transpose(data, (0,2,3,1))\n",
    "    temp = []\n",
    "    for d in data:\n",
    "        for aug in auglist:\n",
    "            d = aug(d)\n",
    "        temp.append(d)\n",
    "    data = nd.stack(*temp)\n",
    "    data = nd.transpose(data, (0,3,1,2))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Residual(nn.Block):\n",
    "    def __init__(self, channels, same_shape=True, **kwargs):\n",
    "        super(Residual, self).__init__(**kwargs)\n",
    "        self.same_shape = same_shape\n",
    "        with self.name_scope():\n",
    "            strides = 1 if same_shape else 2\n",
    "            self.bn1 = nn.BatchNorm()\n",
    "            self.conv1 = nn.Conv2D(channels, kernel_size=3, padding=1, strides=strides)\n",
    "            self.bn2 = nn.BatchNorm()\n",
    "            self.conv2 = nn.Conv2D(channels, kernel_size=3, padding=1)\n",
    "            if not same_shape:\n",
    "                self.conv3 = nn.Conv2D(channels, kernel_size=1, strides=strides)\n",
    "    def forward(self, x):\n",
    "        out = self.conv1(nd.relu(self.bn1(x)))\n",
    "        out = self.conv2(nd.relu(self.bn2(out)))\n",
    "        if not self.same_shape:\n",
    "            x = self.conv3(x)\n",
    "        return out + x\n",
    "\n",
    "class ResNet(nn.Block):\n",
    "    def __init__(self, num_classes, **kwargs):\n",
    "        super(ResNet, self).__init__(**kwargs)\n",
    "        with self.name_scope(): \n",
    "            b1 = nn.Conv2D(16, kernel_size=3, strides=1, padding=1)\n",
    "            b2 = nn.Sequential()\n",
    "            for _ in range(8):\n",
    "                b2.add(Residual(16))\n",
    "            b3 = nn.Sequential()\n",
    "            b3.add(Residual(32, same_shape=False))\n",
    "            for _ in range(7):\n",
    "                b3.add(Residual(32))\n",
    "            b4 = nn.Sequential()\n",
    "            b4.add(Residual(64, same_shape=False))\n",
    "            for _ in range(7):\n",
    "                b4.add(Residual(64))\n",
    "            b5 = nn.Sequential()\n",
    "            b5.add(nn.BatchNorm(),nn.Activation(activation=\"relu\"),\n",
    "                   nn.AvgPool2D(pool_size=8),nn.Dense(num_classes))\n",
    "            self.net = nn.Sequential()\n",
    "            self.net.add(b1, b2, b3, b4, b5)\n",
    "    def forward(self, x):\n",
    "        out = x\n",
    "        for i, b in enumerate(self.net):\n",
    "            out = b(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_pic, train_label, test_pic, test_label = load_cifar()\n",
    "\n",
    "batch_size = 128\n",
    "train_pic = np.transpose(train_pic.astype('float32')/255, (0,2,3,1))\n",
    "test_pic = np.transpose(test_pic.astype('float32')/255, (0,2,3,1))\n",
    "mean = [0.4914, 0.4822, 0.4465]\n",
    "std = [0.2470, 0.2435, 0.2616]\n",
    "for i in range(3):\n",
    "    train_pic[:,:,:,i] = (train_pic[:,:,:,i] - mean[i])/std[i]\n",
    "    test_pic[:,:,:,i] = (test_pic[:,:,:,i] - mean[i])/std[i]\n",
    "train_pic = np.transpose(train_pic, (0,3,1,2))\n",
    "test_pic = np.transpose(test_pic, (0,3,1,2))\n",
    "train_data = gluon.data.DataLoader(gluon.data.ArrayDataset(train_pic, train_label.astype('float32')), \n",
    "                                   batch_size, shuffle=True)\n",
    "test_data = gluon.data.DataLoader(gluon.data.ArrayDataset(test_pic, test_label.astype('float32')), \n",
    "                                  batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aug_train = image.CreateAugmenter(data_shape=(3, 32, 32), rand_crop=True, rand_mirror=True)\n",
    "\n",
    "net = ResNet(10)\n",
    "net.initialize(ctx=ctx, init=init.Xavier())\n",
    "loss = gluon.loss.SoftmaxCrossEntropyLoss()\n",
    "trainer = gluon.Trainer(net.collect_params(), 'nag', {'learning_rate': 0.1, 'momentum': 0.9, 'wd': 5e-4})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 loss:1.4406 tracc:0.4645 teacc:0.5239 time:46.413\n",
      "10 loss:0.5094 tracc:0.8247 teacc:0.7747 time:46.523\n",
      "20 loss:0.4474 tracc:0.8451 teacc:0.7952 time:46.910\n",
      "30 loss:0.4176 tracc:0.8569 teacc:0.8170 time:47.083\n",
      "40 loss:0.4018 tracc:0.8619 teacc:0.7925 time:46.993\n",
      "50 loss:0.3897 tracc:0.8655 teacc:0.8101 time:46.851\n",
      "60 loss:0.3844 tracc:0.8693 teacc:0.7707 time:46.536\n",
      "70 loss:0.3754 tracc:0.8717 teacc:0.8485 time:47.016\n",
      "80 loss:0.2465 tracc:0.9175 teacc:0.9119 time:47.204\n",
      "90 loss:0.0928 tracc:0.9694 teacc:0.9251 time:46.772\n",
      "100 loss:0.0762 tracc:0.9745 teacc:0.9140 time:47.048\n",
      "110 loss:0.0781 tracc:0.9743 teacc:0.9129 time:46.788\n",
      "120 loss:0.0757 tracc:0.9738 teacc:0.9110 time:46.901\n",
      "130 loss:0.0779 tracc:0.9740 teacc:0.9046 time:46.762\n",
      "140 loss:0.0471 tracc:0.9853 teacc:0.9326 time:46.777\n",
      "150 loss:0.0101 tracc:0.9984 teacc:0.9356 time:47.160\n",
      "tracc:0.998889 teacc:0.934731\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8W9WZ//HPI8mSdzteYiexs+9h\nSwiBsIaGJTAdoNMNCu0UKLSlTPsD2hmYTpkOnZn+ukw7v05pKW3pAhQGKIWUUvadFsgCCQnZnMRJ\nnM37bsuWdH5/PFKkOHZsJ3ZsKc/79fJLlnQtHd/E33v03HPPEeccxhhjUotnpBtgjDFm6Fm4G2NM\nCrJwN8aYFGThbowxKcjC3RhjUpCFuzHGpKB+w11E7hORahFZ18fzIiI/EpEKEVkrIguGvpnGGGMG\nYyA9918Dyw7z/CXAjOjXjcBPj75Zxhhjjka/4e6cew2oP8wmlwO/deotIF9Exg1VA40xxgyebwhe\nYwKwK+F+VfSxvT03FJEb0d49WVlZp86ePXsI3t4YY/oXjjhagiE6u8NEIo6Ig4hzhCOOiHNEIhB2\nse8dh7t2X+Cwz/dnQn4GBVn+I/rZVatW1TrnivvbbijCXXp5rNff2zl3L3AvwMKFC93KlSuH4O2N\nMaZvkYjj+89t4t7XthGKOAIeIcvvJTvgIyv6pd979b7fR2bAS8DrweMRvCJ66xEE6OgOEwxFyEjz\nkun3kuH3kub1gAOHHjQAPAKCIAIiovcFPCKcVJbPlKKsI/p9RGTHQLYbinCvAsoT7pcBe4bgdY0x\n5qh0hSL842NreOK9Pfzd/AlcfcZETikfg9fTW580tQxFuC8HbhaRh4HTgSbn3CElGWOMOZYiEcet\nj7zHU2v38rWLZ3HTkmmIpH6ox/Qb7iLyELAEKBKRKuBfgTQA59w9wNPApUAF0A5cO1yNNeZ40x2O\n4BHps6fZFgyR6fcOKLRideRk6bW2dHbTGgxRlB3QsscgfffZTTy1di+3XzKbL5w3bRhaOLr1G+7O\nuav6ed4BXxqyFhmT4pxz1LZ2sbuxg90NHTR1dJMV8NIVilBZ10ZHV4TCbD+b9rXw4ob9hCKOGSXZ\npHk9tAVD5GWkMSbTz8Z9Leysb6coO8C88bkUZQfw+4Sd9e3squ+gvq0Lj8DCyQWkp3l4s6KOju4w\nM0uyyU1PozUYYmxOgLnj86hpCbJudxPtXaEDJ8zSPB5OKc9nVmkOb2+vY2tNG2dMLeCksnxqWoLU\ntXbR0R2msztMe1eInPQ0Zpfm4PMIe5s62dPUyf6mTrweITvdR01LkOrmTiYWZjJvfB4T8jNI83l4\nZ3s9lbVtTCrMpCg7wP7mTiqqW6moacU5rVMXZgUoyQ1QkptOcXaAmtYgm/a10NTRTWd3mPKCTGaX\n5jC7NJeCbD/PrtvHGxW1XH36RD5/7tQR/fceKTJS87nbCVWT6upagzz9/l6e+2A/VQ0d1LYEiThH\nd9jRFY70+jNejxDweWjvCjMmM42L55WSHfCxuboV5xyZfi9NHd3UtASZVpzNvPF57KxvZ+O+Zhrb\n40FXXpBJYZafzu4w71TW09kV5qzpReRnprFxXwsdXWEyAz72NHawtaaVnICPk8ryyctMA3SURHtX\nmJWV9TR3hhifl870khxWVtbT3hUGIMvvJTPgIyPNS0aal/r2LmpaggD4fR7G5aVTmptOOOJoDYYo\nzPZTnB1ge20bG/e1EAzpPijOCTCrJIed9e00tHUxNjfApMIsTinPpzDbT3VzkOqWTvY3B9nX1El1\nS5CibD8zS3IoztFe/Y46fc3Kujacg/KCDD5+ajk3LZmG7wh6/aOZiKxyzi3sb7uhqLkbc9yLRBxv\nb6/ntS01rNhez7baNurbugCYPjabueNzKc4O4IuOuhiXl86EMZlMyM9gTFYabcEwPo8wYUwGaV4P\nHV1h0rxyTIKpszuMPzoypKdQOML+liDj89IRETq7w+xr6mRsboBM/6HxUduq4V6Y5T9sqcg5R0N7\nN23BEGVjMoasFt7eFWJ/c5DJhZnHVX29NxbuxhyBUDjCE+/tYePeZsLO8eqmGrbVtuHzCCeV5bHs\nhFImFmRy7oxi5ozLGXTQZPi9w9TyQ6Wn9f1ePq+HCfkZB207+TBD+IqyAwN6TxGhIMt/xGO9+5Lp\n9zGlyGINLNyNGZTO7jB/WruXu1+pYFtNGxlpXkRgdmkOP/jEySw7obTXHq0xx5r9LzRmAJo6uvn5\na9t44O0dNLZ3M7Mkm599+lQumlty3H/8N6OThbsxh+Gc44G3d/K9ZzbS3Bli2bxSPnPmJBZPLbRQ\nN6OahbsxfWhs7+IfHnqX17fUctb0Qv750jnMG5830s0yZkAs3I3pw09f2cpfttbxrStO4JrTJ1pP\nPZl1NEL9NnAR8PohqxjS0qG7Ux/f+55u4yL6uD8butsh2Ao54yB/IoQ6oKtdfzazAEJBCHXqVySk\nr+tNA08aZBZCwVSo3Qyb/gzhLn2dnFK9LZymrzGMLNyN6UV3OMLvV+/mQ7PH8ukzJo10c8yRqloF\nf7oF9q6l/3kcozN7uYRrEMQLLnx0bRCPfkVC8ccu/T4suuHoXrcfFu7G9OKVTTXUtgb5xMLyQ5+M\nhLXX5s/s/Yc7m8CXAb7oML/2ekjPB48HnNOfTUsfvsabuPcegNotsOR2KD1Re9WhTmirif875E6A\n8fMhe6z+TKgLulohLQN86dC6H5qqIC1TH2urhY4G/ff1ZYAvAB4fRLoh3K2v21YNddu0pz7rUsgY\nA+110LIXWvZB8axh/9Ut3I3pxSMrd1GUHWDJrIRps7vaYd3v4bXvQeMO/XhdMA0Kp0LZaTBtKaz+\nLbz+XxoKkxZD3Vao36ohkF8OzXv04/6ks+DEj8FJV2rAtFZrWaB4ppYC3vxvDY0zvxI/SCRq3Kmh\n5BngeHjn4IMntB2zDrew2mGEu7XskEx2rYDy0zXcB8rnB19CySSnVL9iCqYcWVuyi/Vr3ElH9vOD\nZOFuTA81LUFe2ljN7QsipP35Nu351W2FluhM1uNOgVM+BQ07NLg3PKWhHnPCRyGQA5Vvam11/jXa\n22vcAVPP1zDf+DT88SvwynegbKHWZSPdMO5kaKuD5ip9rfVPwpy/1R5f+SJ97Vf+L7zxA5hwKlz2\nP1AyTz9NvPpdWPM7mH6Bvs/+9dqDzJ8EFS9A5evaw7z2GSg/7fA7oasN3n8U5lymvc7nvwEr7oMr\nfgLzrtBtgi16sMoYE+/1DsaGP8Lzd8LSf42/5lAKtkL1ejj3a0P/2knA5pYxpodHV+xk4xP/l68H\nHsXj88PYOVA4XXvpExbAtA9pbTbGOdj3vgbouJNh+tL+38Q52P4qvPxtqNkAJ18FY6bAew9q7/ji\nb0N7LTx1i5YFArkQbAZ/DnS1wOwPw863tDww4VR9zap3oGwR7FurpQcE0vOgsxECeXD+HfDWT/VA\n8MnfQvVG7fmPmQw1m/TnJ56p7X/4U7B7FWSXwKQzYf0f9PvWajjpk3oCsmajvq8vHc79Kpz5Zf20\n4Zz+7JqHYP0T+v6+DDjnVjj7Ft131Rvg50ujpYwuWHgdXPyfWvZI3EcN22H763qwyZsAU5fo7wR6\n0M2d0Hd5bPtr8Ju/hasfgxkXDu4/wSg20LllLNyN6eH+B+7j0xW34GZeglz2P/pReqSEu+MjPDY/\nA2//THu5p35Wa/l/vVsPEg07YOk3YMFntLxTu1kPSoEc6GzWHrs/E3avhvsu1kDtyZ+ttWbx6Ptd\n8E149wHYvw7OuAmW3glPfFF73BMXw7TzIW8ibPwjfPCkHkDKF2kg11Vo6M+6RA8e+9fDluf0U0zp\nyfDWTzSwP/cCrPgF/OVHMHYefOyX2u6GHfDETbDjjYPbOHYuXP+8Htge/Ji+9hU/1RJY7CC77309\nAL353/DSt+Aftw/7yJRjycLdmCP08//6Z25ouRtu23RwrTVVbHsFmnbreQKAhkod6lc0Eyqeh/d+\nB2d8ESaeoScX967R0lHs00o4BN4eFd2tL2kvfdc7kFWk4Tr3sngvOxKBl+6CN36o9zMK4MrfaSgD\nbHkB/vB5/bRSPFvLPc7Bkn+CGRfr0MLK1+Cx67TktHulnvPo7tByV2ahHsBa9+vrXfxt7bnXVcA/\npFbOWLgbcwTCEcfP/u16viiPI9+oPTTEzNGp366jTrLHHlzaAmjZr+cMtr+u5Z1l39aeeaI3/5/W\n6TPGwI2vQGYRrPyl9vS7WrWEtO5xLRshMPtv9DxBCrEpf405Attr28gLN9KVmU/Agn3oHW6kSU6J\n1uTPvqXvbc78spaMyhbFg/+srxy8TdkiuOcsLWeV9XPiOIWl1iz2xhyl9XuaKJImXNYI1tlN30S0\nZFR2at/blMzVcxKgwyCPU9Y1MSbBut1NXOJpwp83bqSbYo7GRf8OMy7SoD9OWc/dmATrdjczztuC\n50jGbZvRw5+lI3WOYxbuxkQ551i3p4kCmo7sohxjRhEry5hhV9XQTkV1K+fMKMbrEZxzNHeEyAx4\nSYuuEeqco76ti0y/78ASc+GIo7Y1SE1LkLyMNIpzAmze38LGvS2MzQ1QmpfOpn0tbK3R5e38Pg9+\nr0dvfR6KcwKcUpZPKOJ4a1sdjR3dZAe8ZPl9ZKf7mFyYxbi8dFbvbOCZdfvY3xwk1NlKIL1DZ/4z\nJolZuB/n9jZ18OqmGiYXZTGzJIfttW3UtgZZNLmA/Mw0dta3U1nXTsDniX55CaTp99UtQTbubeHt\n7XWsrGxgUmEmF88rpSg7QGd3mM3VLayqbGDljgYAFk0p4MZzpvLjlyt4b1cjAAGfh5z0NILdYVqC\nIdLTPHxo9li6Qo6/bq2lresoZ+TrR3qah87uCH6fh6IsP4tLItCE9dxN0rNwTyLBUJjdDR3kZ/oZ\nk5l2YH7x1TsbeH1zLY0dXRTnBLj69EnkZaRR2xpkW00bNS1BvB4hPzONrTWtrN7RSGc0TN+sqCUc\nOfRaB49AYXaAmpZgv+0qyvazaEoBm/a18K/L1x943O/1MKs0h69eNJP8TD//+fQGPvfblRTnBLjt\nwpkAtARDtHSG8HuFiYVZVNa28cz6fQR8Hi6fP4E543Ipzg7Q3NHNvuZOphRlMW98LjUtQfY1dzJj\nbA4zS7IB6ApH6ApFCIb0dndjx4GDyOKphYzLT6ctGKYtGKK5o5utNa1sqW7lhAl5XHriOLIDPp1o\n6pdAloW7SW52EdMo45yjtrWLbTWt1LZ2UdcWZMPeFt7f3cimfS10h/XfKz3Nwwnj83DAqmjPODvg\nozUYIjfdx8TCTNbtbu71PYqy/eRlpOER4UOzx3LF/Ansaexga00rU4qyyctI4/UtNeysb2fh5AJm\nl+bQHdbQDHaHCYYidHaHGZPpZ3ZpLuUFGQcONDvr2unoDpPmFcoLMg+UXUDHkL++pYaPLigjKzBK\n+xUb/6Tzqtz4ik4Da8woYxcxjZDmzm6efHc308fmMH9iPjvr21lZ2cBLG6vZUddG2ZgMCrMDhMIR\nMgM+phVns7exgxc27Gd/c5Cwc3SFIge9Zm66j5PK8rn+7KlMH5tNS2c3O+vbeb+qiaaObu788Fw+\neVo5WQEf6/c08eOXKqhr7eKrF83kpLJ8inMChCOOhvYuysdkMqkw85BVheaMy2XpnJID9xdNObK5\nOCYW9jGJEzClKIspRVlH9LrHTGu13lrN3SQ5C/ej1NTRzaMrdzF3XC6zSnP4zH3vsH7PoT3mCfkZ\nzBmXy57GDjbtayHN56GxvZumjm78Xg9nTi/kwrkleDxCaW46U4uzKckNMCbTz9icwICXeJs3Po+f\nXnOYCzyOpUgYtr6sE0wNdN7xI9XRoDMRTr/g6F6nrUZvLdxNkrNwH4SK6lZW7ahnT2PngZEZ976+\n7UBdOj3Ng3NwzzULAGH9niamFmdx4oR8phVnHRLQsRJMht+r9d5Us+0VePCjcN4/wfn/PLzv9c4v\n4OX/gNt3QnquzjRYtRI+cs/gXqe1Wie78gWGp53GHCMpmChDp6mjm9ZgiNLcdH715na+88xGusNO\nl1mMnqo4YUIu91yzgIrqVp5au5ebz5/O6VMLAVh2wuFnFBQRinNSOERqNunta9/TOdAnnnH47Tub\ndbrZQHb/r93VDrve1vm9RXT2P5zOKpieCxUvwqan9cAymJVz2qrtZKpJCRbuPXSFIvz2r5U8tqqK\nTftbcA58HiEUcVw4t4R/vnQOZWMyCEccdW1dlOam4/UIp04q4JOnTRzp5o8udRU6x3fmGPj9DXDz\nOwcvxtDTY9fqHOBXPnj4122thoeu1DLMdc/qQaNhuz7XXq+rzrfX6f0PntCJqNb8L2QV9l+2aa2x\nYZAmJVi4R4UjjqfW7uGHz2+msq6dhZPG8H+WzqQox09lbRszSnL4+KllB0oraV6tow+Z176vi+bO\n+duDH+9s0lV4RHTh3fcf01V7PMf44uK6rbqizoRB1PPrtuiaoGd8Uefhrt6gKxn1pXaL/o6H09EA\nv7hAl50DXQRi4hlQv03vx0K9rVZv1/9BVy168iadIbCvcI/NUd5WDSUnDPx3NGaUOm7DvSsUYVf0\nysnVOxt4bv1+tte2MbMkm19fexpLZh3D3lvLPq0XT7/w4HCvfAN+e4WuNHPSx2HtI7D8Zl3ybeJR\nznbnHOx4UwMvsb7c0QCbn4WTrzx4+z9+RRdtuPUDXd0HdJWg+z+iKwOd9rlD36NuK0w5Nz41a2wh\nhb7a01aji0d3NOh83b3Z8oIuznD17+GRz+iKQ8GW+InQWLi31+q84XvXwOM3QiSkB4+e2ut1TdKV\n9+l+bq2BadZzN8nvuJxb5rXNNZxy13Ms/a9X+fz9q/jVG5UUZwe455oFPPOVc4c22FtrdKmySKTv\nbdY9rnNPN1Ul/Fw1PHa9rjG54Ul9bOuLehsLspiKF3StzcFcs/Da9+DXf6M920Svfk9XxGncFX+s\nvR52/EXX8Hz3gfjjq3+riy5vfPrQ1+9qg+bdukB0dvTcQ8u+vtvT1arBDrq2Z18qX9dSz7TzoWiG\n1vUbKhPaWqcHnc4mXUwaYM9qXWmovVYPHDGdTfCTxbDi57qSzzP/BMEmq7mblHDchfuGvc3c9OBq\nJhZk8oNPnMzjN53J2m9exCNfWMyyE8bh8QxsyOGAPf8NePJL8PZP+97m/Uf1NhbuzmnAdjbC5HNg\n6yu6nNjWl/X5WO80Zv0T2vPc+deBtWn1/fpJAXR9zJhQF6x9WL+PlT1A1750YV3WLLbAcrAFXvm2\nPl/9waHvESuTFE6P1rAlHu7rn9BPJIkHo9j4ctAFo2PCIV2hJ3Zw3PGmLs3m8WoZq3azru4T01ar\nByPQxaonLoa8crjg3/Sxuq3xbdc9Dq374JrH4epH4sE/kmumGjNEjpuyjHOOZ9fv55vL15MV8PKr\na09jXN4Q1sx707xXa+RpmfDCNzUcV96noXbtn3Sbuq3as8ydoD3dzibo7tQ1Kc//FyiZBw9fpQsh\nd+ql9LTX9nifPXq74he6zNjhtNdrL3/q+Rq2ib3kTU/HDxyJveyNf9K2L/s2PPpZDfjaTfoJYu7l\nujhye/3BixDHSiCFM8CbputqtkZfs+IF2PYyNO3SHjUcHO7VG3Uf3X+FLrnmwro488mf0pO0sYUY\nimbC2v+NH6D82dr+2P7JKoJP3K8lma7WeLvKohf3vfuALrg8dYme01h4ne5D67mbFDCgnruILBOR\nTSJSISK39/L8RBF5WUTeFZG1InLp0Df1yO2qb+fyu9/kCw+sIsPv5VefXTS0wV6zWXuYPb1zrwbT\nZ5/SGvJj12pZYccb0BYN0fcfBQQWf0nvN+2G+mjvcsJ8rVl7/fD6D3Q7T1q8ZxoT62V/sPzgkOxN\n1Uot9ZxzG5SeEB+uCBp2sQWNY/Xx7k4dVjhzGcy5DPInwXNf15LMKdfAgs/odj1777EecsFUvc0u\n1R44aKgD7Ev41NAWbbc/W19r31odJz/vCl027S8/1k8QAJPP1tviWXq7+Vndv2Mm676JHaAyi7QX\nnjtOn/P49CQv6Mnd3Sth/jXxtTyX3gnnfBWmnHP4fWhMEug33EXEC9wNXALMBa4SkZ7Lm/wL8Ihz\nbj5wJTBqVqRdtaOBK+5+k8raNr7/8ZN5/pZzmTs+d+jeoKkKfnK6ll4SywxdbdpLn/1hHWFy5e90\nrceP/0af3/Ou3m74I0w6SwMs9noHgnGajvmefDZ0t+lcJznjDi3LNO/WnnikG969//DtrXpHx5JP\nWKDh2LQTgq16UNn6op4YFW/8gFH5ur737L/RUsjVj8InH4AvvwuX/xjGztPt9q8/+H3qKiC3DPzR\n6QhySuKv2bhTb/e9H98+dlCadBbUbIQNT2k7L/kuXPQt6KjXTz+BXCg9Sbctiob73vdgzBT95NBe\nFx8pk1kYf31vmgZ8XYXef/cBDfuTPhnfJj0Pln4jfsLYmCQ2kJ77IqDCObfNOdcFPAxc3mMbB8QS\nMw/YM3RNPHLv7Wrk6l+8RXa6jz986Sw+dmoZPq9Hw+W3V8RD5mhsfVlPhq59WHvqMRuf1jLKGTfp\n/bKFcOFdejEPoqWYtlotKUw7H/LKdLumnRpAnjStFQPMuFhvp18QD7CYYKuWcqaeB1POgxX36QlF\n0F5s4glEgKoVWurxZ0HxbH2sdpMeZFwETrlaa+SxXvbWl7SsNDnam40N1yyYqj3enFLIKOg93Aun\nxe/nlOqngUjCieP9PcJdPDD5LC33rHlI6+VZRTrUceJiDfiJi+NTGRRM0YCOfZ9ZGC3LRPdPVtHB\nbSqcDrUVun/WPKwr9fTcxpgUMZBwnwAkDJ2gKvpYom8C14hIFfA08A+9vZCI3CgiK0VkZU1NTW+b\nDJnK2jau//UKxuak89gXzmRaccJVjy/9u9Z8V/3m6N9o2ytao511KTxzR7w3uudd8GUcuvp6eq6O\n8ti9Woc6gpZesks00JuqtCwzZrKOuwata487GU78mAZYW0LNPdYbzp0Ai2+G5iot9XS1wb3nwQ/m\nwnP/okEfCUPVqvinhOI5eluzSevghdOjo1tK4vXx2i36eFp677+/iB4sEssyzmn5o2hG/LHsUg3w\nlr0Q7gLk4LJM634to5REPwk07dJPPTFn36K3sZIMaG+8IHoAGdNLuPccTlk4Xfftpj9rXX7+p3v/\nnYxJAQMJ996Gj/Qcc3cV8GvnXBlwKXC/iBzy2s65e51zC51zC4uLh29EQntXiOt+s4KIc/z62tMO\nvsR/3zodL+7xaQgezZTHkYiG+9QlcNmPtb6+6Rl9bu8arWl7ezlnPX6B9twrX4e0LC23eDyQOz5a\nltl2cK83dxx8/jXtNccCLCZ2MjV3PMy4EEpOhDd+qGO3G3dqb/6vd8PjN2iId7XEDzhjJms9f897\n2pbYBT45CfXxhsr+L98fOxf2fxAf0dJep58mCqfHt8kp1f0TK0eVLdSrSoMter8temXo2ISK35yE\ncJ9xkZa0Fl538HsX67zwB3ruHQ3R+WHyNfwTFc2AUCe8+l092Exbevjfy5gkNpBwrwLKE+6XcWjZ\n5XrgEQDn3F+BdGDEPu/+x582sL22jbuvXsDU4h7zlLx4l/aeL/p3vRimaoUOM0ys/w5U9QfaA5y6\nRC9tL56j851EInpCcNzJvf/chAXaU/3gSR3WFwuhvHIN5Ppt8R5pT1lFB59QjYV7zjjtRZ9ziw4P\n/MuP9ITnpx6Gpf+qPfO//li3jYW716ejWdY+rKE3/UJ9PNZzj4R1H8UuQupLyVytyzfu0Pu73ok+\nPi++TXZ0OuGq6HOxxYv3R3v8rfs13HPG6Tj2cSfHR9KA/m7zrjh03plY3T3Wcyf6qaG3ckth9JPE\n/vfhlKt6P/AakyIGEu4rgBkiMkVE/OgJ0+U9ttkJLAUQkTlouA9v3aUPL23cz4Nv7+SGc6Zy5rSE\nP/DaCnjw47DlWf2If8rVOo/Jql/D/X8H95yj2wzGtlf0dup5elu+SMOrfpte8NNXuI+PXoLfVhOv\nZYPW3feuhVDHwT33RJkF2vuOXabfvFtvc8fr7dwrtB6eMUZr/ACLbtApbN97UB9PfO2xs7WX7UvX\nejdowLbVamkk3KXBeTixk6qx0szmP+uJz/KEicJyxultVXSBllnRAVWxuntrjZa3RHTI5YXfOvx7\nxkw5R8s5Y+fET6DWbD74ZGpMYpnolGsG9vrGJKl+w905FwJuBp4FNqCjYtaLyF0icll0s9uAG0Rk\nDfAQ8Fk3Aks8Oef41lMbmFWSw20XzYw/0dkMP/8Q7HwLLvoPOPPL2nufdYkG3q63ABcfapcoEobl\nX4Znv37oc9tf1d5g7GRo+ekalO8/ovf7CvfSE+MnAqf0CPdQh37fZ7hHQytWmmneoyc0YxNyebzw\nmSfhcy/qpwnQk6exmnXZafGhfxA/qTrprPhr5JQATpecg/577mPnAKIXUUUiOjRx+lLw+ePb5ER7\n7rtXa3uLZ+volH3rtDQW67kDzL86fsDsz9Ql8I9b9aAX2zet+zTwe8oq1vcsPwOKph/6vDEpZECf\nS51zT6MnShMfuzPh+w+As4a2aYO3tqqJ7bVtfOejJxLwJSwOseddvaz8U4/CzIvij596LWx+Di77\nkdaoK16AxTfFn3cO/nQrrI6eeD3lU1pq6GzWC3m2vQIL/j6+fWxK25W/0pOjsROWPaWla225oRJK\nEw4AsYME9F2WSQz33PF6gjK3x/ntxHJGzMLrdATKnMsOfjw2VjxxQq3YdAE7/xJtSz8990C2nvRd\n+WsN29b98Z75gdeMhnuoQ+vkIjpB1/51+iknHDz62RgTe+uZvawkJQIf/eXB+9mYFJVSRcfla/bg\n93pYNm/cwU/sWa23sSsTY6aep4s7eH06feyKX+o84bGx2X/5kZZtFn0e3vudnoi78N/gV3+jo1Jm\nfxjO/Vr89QqmRkezVGuvPbHn2tM5t2rpI7HuGxv66Es/NLBjDum579YTrv1Jy4AvvHHo41PO01km\nT/x4/LFYL3vnW/oJI3cAYXju13R63T98UcfJ95x90RfQHntHffz3HD9fh4/Gpio42itDE8O9ryGO\nMy48uvcwJkmkzNwy4Yjjj2spg1BqAAAS4ElEQVT2cN6sYvIye4yS2L1aSwu99eZi4Tp9qfYed7yp\n94Ot8Pp/6ZWZl3wHTv+8ngD91aV6Kft1z+m847EgBO0Zlkdna+yrJBMz7yNaC08U61GOmdL3lL6x\nckNiWSZWbz8SGfm6WlHifCqxnnv1Bg3igZx4LD1BD3Zt1foJprd9nRN93fxJejtzmdb010TnsxnS\nnruNXzfHt5QJ97e311HdEuTyU3oJuj3vxk9i9mXS2Toufcvzev+932n9/JzbNLQXf0kvje9shk8/\n3veUuwMN997kRXvrfdXbIR5gbXV6UrWtpu9e/pGKTfSF67/enujcr+nPJY5PP+h1owfC/GjPfeJi\nPcG75qGE9z0Kaen6bwS9n1A15jiSMuH+xzV7yPJ7WTq75OAnWmt01MfhFokADYbJZ8PmZzQ43/qJ\nXuxTHr3gJ7NAT1Te8OLhF6yYcaEO5Zt87uB/iUCO1uInLu57m9iFOe118cm9jqbn3htvWjwcB7NE\n3fhT4EvvHPqJJOZAzz16TsDr06tvO5v0fnZJ7z83GLFPDHblqTnOpUy4v76llnNmFJPh9x78RKze\n3l/PHXQSqcad8MN5eoFNbDKvmLJT4ycg+1IyD+7YGb+4ZrBu+iuceXPfz3t9eoFOe93BFzANtdjQ\nxcH03EF/754XDx14zWi45yVcNjE7euJVvFqTP1qxg1JvZSFjjiMpEe57Gjuoauhg0ZRe/qB3r9Y5\nSwZSJpl3hZ50nHy29tr7Ki+MtKwivXgqNsY9ZzjCPdqL7m+M+2AUz9GySeIBY9pSvUo2q3holg48\nEO7WczfHt5QYLbOiUq/YXDSlQOeN8Qbg3K9qrXz3Kh1T3fPKxr6UngDXPDaMrR0CsSkIYnOmD0fP\nPXZSdTBlmf6c+HEdipqeMCtnIFtPrPac4OxIxcLdyjLmOJcS4f729npyAj7mNL6iy8eBht/EM3Q4\n39yek1gmucxCHSO/9mG9wjV9CKcwjokNr4yNbBkKHk/va6N+5Gc6I+VQyCvTslVa5tC8njFJKiXC\n/Z3t9ZxX7sH79G061/eks3RZu7d/qnOPJF6YlAoyC3TVJNDVmobDaZ/TUtZwHDh68g9hEJ/1FR23\nn3gVrjHHoaQP97rWIBXVrfy/rMego1FHtIydq8MJ0/N0kWSPt/8XSiaxenLGGJ1bfTjklA7faw+n\n9Lz4alLGHMeSPtxXVGqtdlr7ezDz4vhMhH0Nx0sFsbryyVf1Pc+6Mea4lvSjZVZU1pPjCxForoxO\nYHUcKJiq0wLEFoo2xpgekr7nXlHdypLCRqQpcvyE+6xL4dYNR39FpzEmZSV9z31XQzvz06NXaiau\n4pPKPB4LdmPMYSV1uDvn2N3QwWxPlU6x29c0ucYYc5xJ6nCvaQkSDEUoD+3Q9ToPN8WuMcYcR5I6\n3Hc1tANQ1L71+Km3G2PMACR3uNd3kEknGW1Vx0+93RhjBiCpw72qoZ0ZUqV3rOdujDEHJHW476rv\nYEHGfr1j4W6MMQckd7g3tHNKYI+uOTrYeceNMSaFJX24n+Q2Q+mJqTd/jDHGHIWkDfdQOEJzYwOT\nOjfClCNY0s4YY1JY0ob7vuZO5rMRD2Gd09wYY8wBSRvuu+o7OMPzARGPH8pPH+nmGGPMqJK84d7Q\nzpme9XSVLhjaxR6MMSYFJG2419TsZ55U4pu+ZKSbYowxo07STvk7pnoFXnEw1U6mGmNMT0nbcy9t\nXksXPihbONJNMcaYUSdpwz29q45GyQdfYKSbYowxo07yhnuoiVZv7kg3wxhjRqWkDffMUDPtFu7G\nGNOrpA337HAT7b78kW6GMcaMSskb7pFmgmkW7sYY05sBhbuILBORTSJSISK397HNJ0TkAxFZLyK/\nG9pm9hAJk0sr3QELd2OM6U2/49xFxAvcDVwIVAErRGS5c+6DhG1mAHcAZznnGkRk7HA1GIDOJjw4\nQoExw/o2xhiTrAbSc18EVDjntjnnuoCHgct7bHMDcLdzrgHAOVc9tM08WLitTm/TLdyNMaY3Awn3\nCcCuhPtV0ccSzQRmisibIvKWiCzr7YVE5EYRWSkiK2tqao6sxUBnU/TYkVFwxK9hjDGpbCDhLr08\n5nrc9wEzgCXAVcAvROSQgrhz7l7n3ELn3MLi4uLBtvWArpZa/Sar8IhfwxhjUtlAwr0KKE+4Xwbs\n6WWbJ51z3c657cAmNOyHRXc03H0W7sYY06uBhPsKYIaITBERP3AlsLzHNk8A5wOISBFaptk2lA1N\nFG7Vmrs3u2i43sIYY5Jav+HunAsBNwPPAhuAR5xz60XkLhG5LLrZs0CdiHwAvAx8zTlXN1yNjrTX\n0eW8pGfaFarGGNObAU3565x7Gni6x2N3JnzvgFujX8OvvZ5GcshKTzsmb2eMMckmKa9Q9XTU0+Cy\nyQp4R7opxhgzKiVluHuDDTSSTaY/adcaMcaYYZWU4Z4WbKTe5ZBl4W6MMb1KynD3dzXS4LLJtLKM\nMcb0KvnC3TnSQ420eHJJ8yZf840x5lhIvnQMtuB1Ydq8eSPdEmOMGbWSL9zbdfh8h4W7Mcb0KfnC\nvaMegKDf5nI3xpi+JF+4tzcA0GULdRhjTJ+SMNy1LBPy21zuxhjTl+QL92hZJpJhPXdjjOlL8oV7\nwTSe9Z6LpFu4G2NMX5LvEs+ZF3EHwqXp/pFuiTHGjFrJ13MH2oIhm3rAGGMOI+nCPRSOEAxFyApY\nuBtjTF+SLtzbusIAZPptXhljjOlL0oV7e1cIwHruxhhzGEkX7m1BC3djjOlPEoa7lmWyrCxjjDF9\nSr5wj5ZlbBUmY4zpW9KFe3us524LdRhjTJ+SLtzb7ISqMcb0K/nC/UDN3cLdGGP6knThHhsKaeun\nGmNM35Iu3OeNz+OGc6aQmWbhbowxfUm62sbiaYUsnlY40s0wxphRLel67sYYY/pn4W6MMSnIwt0Y\nY1KQhbsxxqQgC3djjElBFu7GGJOCLNyNMSYFWbgbY0wKsnA3xpgUNKBwF5FlIrJJRCpE5PbDbPcx\nEXEisnDommiMMWaw+g13EfECdwOXAHOBq0Rkbi/b5QBfBt4e6kYaY4wZnIH03BcBFc65bc65LuBh\n4PJetvsW8F2gcwjbZ4wx5ggMJNwnALsS7ldFHztAROYD5c65pw73QiJyo4isFJGVNTU1g26sMcaY\ngRlIuEsvj7kDT4p4gB8Ct/X3Qs65e51zC51zC4uLiwfeSmOMMYMykHCvAsoT7pcBexLu5wAnAK+I\nSCVwBrDcTqoaY8zIGUi4rwBmiMgUEfEDVwLLY08655qcc0XOucnOucnAW8BlzrmVw9JiY4wx/eo3\n3J1zIeBm4FlgA/CIc269iNwlIpcNdwONMcYM3oBWYnLOPQ083eOxO/vYdsnRN8sYY8zRsCtUjTEm\nBVm4G2NMCrJwN8aYFGThbowxKcjC3RhjUpCFuzHGpCALd2OMSUEW7sYYk4Is3I0xJgVZuBtjTAqy\ncDfGmBRk4W6MMSnIwt0YY1KQhbsxxqQgC3djjElBFu7GGJOCLNyNMSYFWbgbY0wKsnA3xpgUZOFu\njDEpyMLdGGNSkIW7McakIAt3Y4xJQRbuxhiTgizcjTEmBVm4G2NMCrJwN8aYFGThbowxKcjC3Rhj\nUpCFuzHGpCALd2OMSUEW7sYYk4Is3I0xJgVZuBtjTAoaULiLyDIR2SQiFSJyey/P3yoiH4jIWhF5\nUUQmDX1TjTHGDFS/4S4iXuBu4BJgLnCViMztsdm7wELn3EnAY8B3h7qhxhhjBm4gPfdFQIVzbptz\nrgt4GLg8cQPn3MvOufbo3beAsqFtpjHGmMEYSLhPAHYl3K+KPtaX64E/9/aEiNwoIitFZGVNTc3A\nW2mMMWZQBhLu0stjrtcNRa4BFgLf6+1559y9zrmFzrmFxcXFA2+lMcaYQfENYJsqoDzhfhmwp+dG\nInIB8HXgPOdccGiaZ4wx5kgMpOe+ApghIlNExA9cCSxP3EBE5gM/Ay5zzlUPfTONMcYMRr/h7pwL\nATcDzwIbgEecc+tF5C4RuSy62feAbOBREXlPRJb38XLGGGOOgYGUZXDOPQ083eOxOxO+v2CI22WM\nMeYo2BWqxhiTgizcjTEmBVm4G2NMCrJwN8aYFGThbowxKcjC3RhjUpCFuzHGpCALd2OMSUEW7sYY\nk4Is3I0xJgVZuBtjTAqycDfGmBRk4W6MMSnIwt0YY1KQhbsxxqQgC3djjElBFu7GGJOCLNyNMSYF\nWbgbY0wKsnA3xpgUZOFujDEpyMLdGGNSkIW7McakIAt3Y4xJQRbuxhiTgizcjTEmBVm4G2NMCrJw\nN8aYFGThbowxKcjC3RhjUpCFuzHGpCALd2OMSUEW7sYYk4Is3I0xJgVZuBtjTAoaULiLyDIR2SQi\nFSJyey/PB0Tkf6PPvy0ik4e6ocYYYwau33AXES9wN3AJMBe4SkTm9tjseqDBOTcd+CHwnaFuqDHG\nmIEbSM99EVDhnNvmnOsCHgYu77HN5cBvot8/BiwVERm6ZhpjjBkM3wC2mQDsSrhfBZze1zbOuZCI\nNAGFQG3iRiJyI3Bj9G6riGw6kkYDRT1fexQZrW2zdg3OaG0XjN62WbsG70jaNmkgGw0k3Hvrgbsj\n2Abn3L3AvQN4z8M3SGSlc27h0b7OcBitbbN2Dc5obReM3rZZuwZvONs2kLJMFVCecL8M2NPXNiLi\nA/KA+qFooDHGmMEbSLivAGaIyBQR8QNXAst7bLMc+Pvo9x8DXnLOHdJzN8YYc2z0W5aJ1tBvBp4F\nvMB9zrn1InIXsNI5txz4JXC/iFSgPfYrh7PRDEFpZxiN1rZZuwZntLYLRm/brF2DN2xtE+tgG2NM\n6rErVI0xJgVZuBtjTApKunDvbyqEY9iOchF5WUQ2iMh6EflK9PECEXleRLZEb8eMUPu8IvKuiDwV\nvT8lOjXEluhUEf4Rale+iDwmIhuj+27xaNhnInJL9N9xnYg8JCLpI7HPROQ+EakWkXUJj/W6f0T9\nKPq3sFZEFoxA274X/bdcKyJ/EJH8hOfuiLZtk4hcfCzblfDcV0XEiUhR9P4x22d9tUtE/iG6T9aL\nyHcTHh/a/eWcS5ov9ITuVmAq4AfWAHNHqC3jgAXR73OAzej0DN8Fbo8+fjvwnRFq363A74Cnovcf\nAa6Mfn8P8MURatdvgM9Fv/cD+SO9z9CL8LYDGQn76rMjsc+Ac4EFwLqEx3rdP8ClwJ/R60zOAN4e\ngbZdBPii338noW1zo3+fAWBK9O/We6zaFX28HB0IsgMoOtb7rI/9dT7wAhCI3h87XPtrWP+jDsPO\nWgw8m3D/DuCOkW5XtC1PAhcCm4Bx0cfGAZtGoC1lwIvAh4Cnov+RaxP+CA/aj8ewXbnREJUej4/o\nPiN+hXUBOoLsKeDikdpnwOQegdDr/gF+BlzV23bHqm09nvsI8GD0+4P+NqMhu/hYtgudCuVkoDIh\n3I/pPuvl3/IR4IJethvy/ZVsZZnepkKYMEJtOSA6C+Z84G2gxDm3FyB6O3YEmvTfwD8Ckej9QqDR\nOReK3h+p/TYVqAF+FS0Z/UJEshjhfeac2w18H9gJ7AWagFWMjn0Gfe+f0fb3cB3aK4YRbpuIXAbs\nds6t6fHUSO+zmcA50XLfqyJy2nC1K9nCfUDTHBxLIpIN/B74P8655pFsS7Q9HwaqnXOrEh/uZdOR\n2G8+9GPqT51z84E2tMwwoqI17MvRj8PjgSx0FtSeRtu44dHy74qIfB0IAQ/GHupls2PSNhHJBL4O\n3Nnb0708diz3mQ8Yg5aEvgY8IiIyHO1KtnAfyFQIx4yIpKHB/qBz7vHow/tFZFz0+XFA9TFu1lnA\nZSJSic7g+SG0J58fnRoCRm6/VQFVzrm3o/cfQ8N+pPfZBcB251yNc64beBw4k9Gxz6Dv/TMq/h5E\n5O+BDwNXu2hNYYTbNg09UK+J/h2UAatFpHSE20X0/R936h3003XRcLQr2cJ9IFMhHBPRo+0vgQ3O\nuR8kPJU4FcPfo7X4Y8Y5d4dzrsw5NxndPy85564GXkanhhiRdkXbtg/YJSKzog8tBT5ghPcZWo45\nQ0Qyo/+usXaN+D6L6mv/LAc+Ex0BcgbQFCvfHCsisgz4J+Ay51x7wlPLgStFF/KZAswA3jkWbXLO\nve+cG+ucmxz9O6hCBz/sY+T32RNohwsRmYkOKqhlOPbXcJ1IGMYTFJeiI1O2Al8fwXacjX5sWgu8\nF/26FK1vvwhsid4WjGAblxAfLTM1+p+lAniU6Nn6EWjTKcDK6H57Av2IOuL7DPg3YCOwDrgfHbVw\nzPcZ8BBa9+9GQ+n6vvYP+lH+7ujfwvvAwhFoWwVaK479DdyTsP3Xo23bBFxyLNvV4/lK4idUj9k+\n62N/+YEHov/PVgMfGq79ZdMPGGNMCkq2sowxxpgBsHA3xpgUZOFujDEpyMLdGGNSkIW7McakIAt3\nY4xJQRbuxhiTgv4/bUTknZaYZ5UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd4e1fc1550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "epochs = 160\n",
    "\n",
    "a, b = [], []\n",
    "for epoch in range(epochs):\n",
    "    if epoch == 80:\n",
    "        trainer.set_learning_rate(0.01)\n",
    "    if epoch == 140:\n",
    "        trainer.set_learning_rate(0.001)\n",
    "    train_loss = 0.\n",
    "    train_acc = 0.\n",
    "    start = time()\n",
    "    for data, label in train_data:\n",
    "        data = augment(data, aug_train).as_in_context(ctx)\n",
    "        label = label.as_in_context(ctx)\n",
    "        with autograd.record():\n",
    "            output = net(data)\n",
    "            l = loss(output, label)\n",
    "        l.backward()\n",
    "        trainer.step(batch_size)\n",
    "        train_loss = train_loss + nd.mean(l).asscalar()\n",
    "        train_acc = train_acc + accuracy(output, label)\n",
    "    test_acc = evaluate_accuracy(test_data, net, ctx)\n",
    "    \n",
    "    if epoch%10 == 0:\n",
    "        print(epoch, 'loss:%.4f tracc:%.4f teacc:%.4f time:%.3f'%(\n",
    "            train_loss/len(train_data), train_acc/len(train_data), test_acc, time()-start)) \n",
    "    a.append(train_acc/len(train_data))\n",
    "    b.append(test_acc)\n",
    "\n",
    "print('tracc:%f teacc:%f'%(train_acc/len(train_data), test_acc))\n",
    "plt.plot(np.arange(epochs), a, np.arange(epochs), b)\n",
    "plt.ylim(0,1)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
